# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from DataSpider.items import GuokrItemLoader,GuokrItem
from utils.common import get_md5
from datetime import datetime

class GuokrSpider(CrawlSpider):
    name = 'guokr'
    allowed_domains = ['www.guokr.com']
    start_urls = ['https://www.guokr.com/ask/']
    # start_urls = ['http://ip.tool.chinaz.com/']

    rules = (
        # Rule(LinkExtractor(allow=r''), callback='parse_item', follow=True),
        Rule(LinkExtractor(allow=r'highlight/.*'), follow=True),
        Rule(LinkExtractor(allow=r'question/\d+'), callback='parse_item', follow=True),
        Rule(LinkExtractor(deny=r'question/\d+/?page.*'), follow=True),
        Rule(LinkExtractor(deny=r'ask/i/.*'), follow=True),
        Rule(LinkExtractor(deny=r'question/\d+/history'),  follow=True),


    )
    # def parse(self, response):
    #     pass

    def parse_item(self, response):
        item_loader = GuokrItemLoader(item=GuokrItem(), response=response)
        item_loader.add_value("url", response.url)
        item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_css("title", "#articleTitle ::text")
        item_loader.add_css("content", "#questionDesc ::text")
        item_loader.add_css("tags", ".tag ::text")
        item_loader.add_value("origin", "random_origin")
        item_loader.add_value("origin_url", response.url)
        item_loader.add_css("create_time", ".answer-date ::text")
        item_loader.add_css("answer_content", ".answer-txt ::text")
        item_loader.add_value("crawl_time", datetime.now())
        questionItemLoader = item_loader.load_item()

        return questionItemLoader
